/*
* * * * * * * * * * * * * * *
*	author: 马丰彦
*	date: 2019/1/12
*	function: 一顿操作猛如虎，猛然发现自己250
* * * * * * * * * * * * * * *
*/

require('../connect');
var Crawler = require('crawler'); // node-crawler
var client = require('../utils/redis');
var url = require('../model/moreLinks');
var repeat = require('../model/repeat');
const firms = encodeURI('电影'); // 电影
var firmUrl = `https://movie.douban.com/j/new_search_subjects?sort=U&range=0,10&tags=${firms}&` // 前缀
// 电影分类 22
var firmType = [];
const gut = encodeURI('剧情'); // 剧情
const comedy = encodeURI('喜剧');
const action = encodeURI('动作');
const love = encodeURI('爱情');
const science = encodeURI('科幻');
const animation = encodeURI('动画');
const sciences = encodeURI('悬疑');
const panic = encodeURI('惊悚');
const terror = encodeURI('恐怖');
const crime = encodeURI('犯罪');
const sex = encodeURI('同性');
const music = encodeURI('音乐');
const dance = encodeURI('歌舞');
const memoirist = encodeURI('传记');
const history = encodeURI('历史');
const war = encodeURI('战争');
const west = encodeURI('西部');
const queer = encodeURI('奇幻');
const adventure = encodeURI('冒险');
const disaster = encodeURI('灾难');
const wuxia = encodeURI('武侠');
const sexy = encodeURI('情色');
firmType.push(gut, comedy, action, love, science, animation, sciences, panic, terror, crime, sex, music, dance, memoirist, history, war, west, queer, adventure, disaster, wuxia, sexy);

// 电影国家 21
var firmCountry = [];
const china = encodeURI('中国大陆');
const usa = encodeURI('美国');
const hongkong = encodeURI('香港');
const taiwan = encodeURI('台湾');
const japan = encodeURI('日本');
const korea = encodeURI('韩国');
const england = encodeURI('英国');
const france = encodeURI('法国');
const germany = encodeURI('德国');
const italy = encodeURI('意大利');
const spain = encodeURI('西班牙');
const india = encodeURI('印度');
const tailand = encodeURI('泰国');
const russia = encodeURI('俄罗斯');
const yilang = encodeURI('伊朗');
const canada = encodeURI('加拿大');
const australia = encodeURI('澳大利亚');
const ireland = encodeURI('冰岛');
const sweden = encodeURI('瑞典');
const brazil = encodeURI('巴西');
const denmark = encodeURI('丹麦');
firmCountry.push(china, usa, hongkong, taiwan, japan, korea, england, france, germany, italy, spain, india, tailand, russia, yilang, canada, australia, ireland, sweden, brazil, denmark);

// 电影年代 9
var firmYear = [];
const earlier = '1,1959'; // 更早
const sixty = '1960,1969'; // 60年代
const seventy = '1970,1979'; // 70年代
const eighty = '1980,1989'; // 80年代
const ninety = '1990,1999'; // 90年代
const twoZero = '2000,2009'; // 2000
const twoZeroOne = '2010,2019'; // 2000
// const twoZeroSeven = '2017,2017'; // 2017
// const twoZeroEight = '2018,2018';
firmYear.push(earlier, sixty, seventy, eighty, ninety, twoZero, twoZeroOne);

var guts = [];
for (let i = 0; i < firmType.length; i++) {
  for (let j = 0; j < firmCountry.length; j++) {
    for (let k = 0; k < firmYear.length; k++) {
      for (let p = 920; p < 2000; p = p + 20) {
        var obj = {
          url: `${firmUrl}start=${p}&genres=${firmType[i]}&countries=${firmCountry[j]}&year_range=${firmYear[k]}`
        }
        guts.push(obj);
      }
    }
  }
}

// 打乱数组
function shuffle(arr) {
  let i = arr.length;
  while (i) {
    let j = Math.floor(Math.random() * i--);
    [arr[j], arr[i]] = [arr[i], arr[j]];
  }
  return arr;
}

client.lrange('list', 0, -1, function (err, arr) {
  var nums = 0;
  var linksss = '';
  var check = new Crawler({
    maxConnections: 10,
    jQuery: false,
    // proxy: 'http://218.57.146.212:8888',
    callback: async (error, res, done) => {
      if (error) {
        console.log(error);
      } else {
        // console.log(res.body);
        if (res.body.indexOf('检测到有异常请求从你的') >= 0) {
          console.log('被封掉了');
          res.options.proxy = 'http://218.57.146.212:8888';
        } else {
          // console.log(res.options.uri);
          var data = JSON.parse(res.body);
          if (data.data.length !== 0) {
            // console.log(res.options.uri)
            await repeat.updateOne({
              url: res.options.uri
            }, {
              url: res.options.uri
            }, {
              upsert: true
            }, function (error, docs) {
              if (error) {
                console.log(error);
              } else {
                data.data.forEach(function (item) {
                  url.updateOne({
                    url: item.url
                  }, {
                    url: item.url
                  }, {
                    upsert: true
                  }, function (error, docs) {
                    if (error) {
                      console.log(error);
                    } else {
                      if (docs.upserted !== undefined) {
                        console.log('成功' + nums);
                        nums++;
                      } else {
                        console.log('重复数据，不存储');
                      }
                    }
                  })
                })
              }
            })
          } else {
            console.log('data没有数据');
            var arr = guts.indexOf(res.options.uri);
            var startStr = res.options.uri.indexOf('start=') + 6;
            var endStr = res.options.uri.indexOf('&genres');
            var startText = res.options.uri.substr(0, startStr);
            var endText = res.options.uri.substr(endStr);
            var endArr = startText + 900 + endText;
            var endIndex = guts.indexOf(endArr);
            guts.splice(arr, endIndex);
          }
        }
      }
      done();
    }
  })

  var a = 0;
  console.log(guts.length);
  var shuff = shuffle(guts);
  function searchUrl() {
    var random = Math.floor(Math.random() * arr.length);
    var host = JSON.parse(arr[random]).host
    var port = JSON.parse(arr[random]).port
    linksss = 'http://' + host + ':' + port;
    repeat.findOne({
      url: shuff[a].url
    }, function (error, docs) {
      if (error) {
        console.log(error);
      } else {
        if (docs === null) {
          check.queue(shuff[a].url);
        } else {
          console.log('已经爬过的循环了');
        }
      }
    })
    a++;
  }
  var myInterval = setInterval(searchUrl, 1);

  function stopInterval() {
    clearInterval(myInterval);
  }
  // setTimeout(stopInterval, 191268);
  check.on('drain', function () {
    stopInterval()
    console.log('完事！');
  });
})